## Pixel explanations for a deep neural network diabetic retinopathy classification modelLet's import some libraries required for the notebook to work:Let's import some libraries required for the notebook to work:
import randomimport osimport torch import torch.nn as nn import torch.nn.functional as F import torchvision.transforms as transforms import torchvision.datasets as datasets import numpy as np import quadratic_weighted_kappa as qwk import transforms_extra import models import matplotlibimport matplotlib.pyplot as pltimport math%matplotlib inline import visualization_rev1 as visLet's define some constants related with the model:Let's define some constants related with the model:
workers = 4batch_size = 1mean = [0.383, 0.266, 0.190]std = [0.291, 0.211, 0.171]normvect = [-mean[0]/std[0], -mean[1]/std[1], -mean[2]/std[2]]input_size = 640input_channels = 3# number of rotated configurationsconfigurations = 5hidden_features = 64And now, let's load the test dataset:And now, let's load the test dataset:
# Data loading code #testdir = '../input/640_itaka'testdir = '../input/640_messidor'#testdir = "../input/640_test"normalize = transforms.Normalize(mean=mean, std=std) dataset_test = datasets.ImageFolder(testdir, transforms.Compose([ transforms.Resize(input_size), transforms.ToTensor(), normalize, transforms_extra.CenteredCircularMaskTensor(input_size, normvect),]))Let's load the previously trained model:Let's load the previously trained model:
# load the modelresume = 'models/ret6_bn/model_best-QWKval0814.pth.tar'print("=> loading checkpoint '{}'".format(resume))checkpoint = torch.load(resume)start_epoch = checkpoint['epoch']best_prec1 = checkpoint['best_prec1']import modelsmodel = models.ret6_bn()model.load_state_dict(checkpoint['state_dict'])print("=> loaded checkpoint '{}' (epoch {})".format(resume, checkpoint['epoch']))We instantiate the previously defined model with the parameters of the pretrained one loaded before:We instantiate the previously defined model with the parameters of the pretrained one loaded before:
me = models.model_explainable(model)#me = me.cuda()me = me.cpu()me.eval()Now, we load a image from the dataset. This image is the one that will be analyzed:Now, we load a image from the dataset. This image is the one that will be analyzed:
dataset = dataset_test#image_nr = 9620image_nr = 999#image_nr = int(len(dataset.imgs)*random.random())batch = torch.zeros(1, input_channels, input_size, input_size)target = torch.zeros(1).long()img, img_target = dataset.__getitem__(image_nr)path, _ = dataset.imgs[image_nr]file = os.path.basename(path)batch[0].copy_(img)if next(me.parameters()).is_cuda: batch = batch.cuda()with torch.no_grad(): input = torch.autograd.Variable(batch) target[0] = img_targetLet's visualize the image:Let's visualize the image:
input_rgb = batch.clone()#print(input_rgb.size())for i in range(3): input_rgb[0][i].mul_(std[i]) input_rgb[0][i].add_(mean[i])input_rgb[torch.lt(input_rgb, 0)] = 0.0input_rgb[torch.gt(input_rgb, 1)] = 1.0vis.plot(input_rgb)#del input_rgb
Let's calculate with the model the predicted classification class.We also calculate the intermediate values of the activations of every layerLet's calculate with the model the predicted classification class. We also calculate the intermediate values of the activations of every layer
#output, features, rf637, rf509_p, rf381, \#rf253_p, rf189, rf125_p, rf93, rf61_p, rf45, \#rf29_p, rf21, rf13_p, rf9, rf5_p, rf3layer_vals = me.forward(input)layer = 17output = layer_vals[layer]print("Image nr:" + str(image_nr))print('File: ' + dataset.imgs[image_nr][0])imagename = os.path.splitext(os.path.basename(dataset.imgs[image_nr][0]))[0]print("Image name: " + imagename)print("Tag")print('Output before softmax: ')print(output)prob_pred_class, pred_class = output.max(1)pred_class = pred_class[0]print("Predicted class: " + str(pred_class))print('Target: ')print(target)## Last layer feature activationsAnd now we plot the feature activations in the last layer of the model previous to the output layer:And now we plot the feature activations in the last layer of the model previous to the output layer:
layer = 16features = layer_vals[layer]score_features = torch.mul(me.lc[0].weight, features.expand(5,64))objects = [str(i) for i in range(64)]y_pos = np.arange(len(objects))for i in range(5): feat = score_features.data[i].tolist() fig = plt.figure(figsize=(20,10)) fig.add_subplot(5, 1, i+1) plt.bar(y_pos, feat, align='center', alpha=0.5) plt.title('Scores of class {}'.format(i))#plt.xticks(y_pos, objects)#plt.ylabel('f')plt.show()
We create a variable to sum the constants contributions of every layer mapped into the input space. Finally we will add this value to the values calculated for the input spaceWe create a variable to sum the constants contributions of every layer mapped into the input space. Finally we will add this value to the values calculated for the input space
with torch.no_grad(): score_accumulated_k = torch.autograd.Variable(torch.zeros(5, 1, input_size, input_size))## Score propagation through average pooling layer:layer = 15rf637 = layer_vals[layer]averager = rf637.sum(3).sum(2).unsqueeze(2).unsqueeze(3).expand(5,64,4,4) + 1e-6 # adding a epsilon to avoid nansscore_rf637 = score_features.unsqueeze(2).unsqueeze(3).expand(5,64,4,4) * rf637.expand(5,64,4,4) / averagervis.stats(score_rf637)#print(score_rf637.sum(3).sum(2).sum(1) + me.lc[0].bias)score_rf637## Scores of the individual feature maps of the 15th layervis.plot_scores(img_target,score_rf637)
#vis.plot_scores(img_target,score_rf637.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf637.sum(1).unsqueeze(0), figsize=(50,50))
## Aggregated scores of 15th layer mapped into pixel space using a 2d-normal 2std=rf priortmp = vis.map_scores_to_input_sz(score_rf637.sum(1).unsqueeze(1), 637)vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))#vis.plot_scores(img_target,tmp, figsize=(50,50))print(tmp.view(-1).sum())del tmp
## Score propagation through classification layer (2x2 convolution)layer = 14rf509_p = layer_vals[layer]rf509, mp_idx = F.max_pool2d(rf509_p, 2, 2, return_indices = True) # 1x64x5x5score_rf509, score_k637 = vis.propagate_score_through_conv2d_sz2_pad0(score_rf637, rf509, rf637, me.rf637)#del val, rf509_pvis.stats(score_rf509)# sum of constant scores of the layerscore_k637 = score_k637.sum(1).unsqueeze(1)print(score_k637.size())vis.plot_scores(0,score_k637.sum(1).unsqueeze(0), figsize=(50,50))#vis.plot_scores(img_target,score_k637, figsize=(50,50))
In the previous map, every pixel corresponds to a receptive field. There is superposition between the activations showed in the above map. We now from bibliography that the receptive fields have a nearly gaussian detection capability. Considering a 2 stdev gaussian model over the receptive field, we can map the activations to input space. The result is shown below:In the previous map, every pixel corresponds to a receptive field. There is superposition between the activations showed in the above map. We now from bibliography that the receptive fields have a nearly gaussian detection capability. Considering a 2 stdev gaussian model over the receptive field, we can map the activations to input space. The result is shown below:
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k637, 637)vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)print(score_k637.sum())print(score_kmapped.sum())#del score_k637
vis.plot_scores(img_target,score_rf509)
#vis.plot_scores(img_target,score_rf509.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf509.sum(1).unsqueeze(0), figsize=(50,50))
## Aggregated feature map scores of the 14th layer mapped into input spacetmp = vis.map_scores_to_input_sz(score_rf509.sum(1).unsqueeze(1), 509)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf509, mp_idx)#del score_rf509, mp_idxlayer = 13rf381 = layer_vals[layer]score_rf381, score_k509 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf381, rf509_p, me.rf509)#del score_mp, rf381vis.stats(score_rf381)vis.stats(score_k509)score_k509 = score_k509.sum(1).unsqueeze(1)print(score_k509.size())#vis.plot_scores(img_target, score_k509, figsize=(50,50))vis.plot_scores(0,score_k509.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k509, 509)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k509print(score_k509.sum())print(score_kmapped.sum())
vis.plot_scores(img_target, score_rf381)
#vis.plot_scores(img_target,score_rf381.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf381.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf381.sum(1).unsqueeze(1), 381)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
layer = 12rf253_p = layer_vals[layer]rf253, mp_idx = F.max_pool2d(rf253_p, 2, 2, return_indices = True) # 1x64x5x5#del rf253_pscore_rf253, score_k381 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf381, rf253, rf381, me.rf381)#del score_rf381, valvis.stats(score_rf253)score_k381 = score_k381.sum(1).unsqueeze(1)print(score_k381.size())#vis.plot_scores(img_target, score_k381, figsize=(50,50))vis.plot_scores(0,score_k381.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k381, 381)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k381
vis.plot_scores(img_target,score_rf253)
#vis.plot_scores(img_target,score_rf253.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf253.sum(1).unsqueeze(0), figsize=(50,50))
score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf253, mp_idx)#del score_rf253, mp_idxlayer = 11rf189 = layer_vals[layer]score_rf189, score_k253 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf189, rf253_p, me.rf253)#del score_mp, rf189vis.stats(score_rf189)score_k253 = score_k253.sum(1).unsqueeze(1)print(score_k253.size())#vis.plot_scores(img_target, score_k253, figsize=(50,50))vis.plot_scores(0,score_k253.sum(1).unsqueeze(0), figsize=(50,50))
#print(score_rf253.view(-1).sum()) # score conservation through mapping to input space testtmp = vis.map_scores_to_input_sz(score_rf253.sum(1).unsqueeze(1), 253)#print(tmp.view(-1).sum()) # score conservation through mapping to input space test (2)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k253, 253)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k253
vis.plot_scores(img_target,score_rf189)
#vis.plot_scores(img_target,score_rf189.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf189.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf189.sum(1).unsqueeze(1), 189)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0, tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
layer = 10rf125_p = layer_vals[layer]rf125, mp_idx = F.max_pool2d(rf125_p, 2, 2, return_indices = True) # 1x64x5x5#del rf125_pscore_rf125, score_k189 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf189, rf125, rf189, me.rf189)#del score_rf189, layer = 10rf125 = layer_vals[layer]vis.stats(score_rf125)score_k189 = score_k189.sum(1).unsqueeze(1)print(score_k189.size())#vis.plot_scores(img_target, score_k189, figsize=(50,50))vis.plot_scores(0,score_k189.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k189, 189)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50)vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k189
vis.plot_scores(img_target,score_rf125)
#vis.plot_scores(img_target,score_rf125.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf125.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf125.sum(1).unsqueeze(1), 125)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf125, mp_idx)#del score_rf125, mp_idxlayer = 9rf93 = layer_vals[layer]score_rf93, score_k125 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf93, rf125_p, me.rf125)#del score_mp, rf93vis.stats(score_rf93)score_k125 = score_k125.sum(1).unsqueeze(1)print(score_k125.size())#vis.plot_scores(img_target, score_k125, figsize=(50,50))vis.plot_scores(0,score_k125.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k125, 125)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k125
vis.plot_scores(img_target,score_rf93)
#vis.plot_scores(img_target,score_rf93.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf93.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf93.sum(1).unsqueeze(1), 93)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
layer = 8rf61_p = layer_vals[layer]rf61, mp_idx = F.max_pool2d(rf61_p, 2, 2, return_indices = True) # 1x64x5x5score_rf61, score_k93 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf93, rf61, rf93, me.rf93)#del score_rf93, valvis.stats(score_rf61)score_k93 = score_k93.sum(1).unsqueeze(1)print(score_k93.size())#vis.plot_scores(img_target, score_k93, figsize=(50,50))vis.plot_scores(0,score_k93.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k93, 93)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k93
vis.plot_scores(img_target,score_rf61)
#vis.plot_scores(img_target,score_rf61.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf61.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf61.sum(1).unsqueeze(1), 61)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf61, mp_idx)#del score_rf61, mp_idxlayer = 7rf45 = layer_vals[layer]score_rf45, score_k61 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf45, rf61_p, me.rf61)#del score_mp, rf45vis.stats(score_rf45)score_k61 = score_k61.sum(1).unsqueeze(1)print(score_k61.size())#vis.plot_scores(img_target, score_k61, figsize=(50,50))vis.plot_scores(0,score_k61.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k61, 61)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k61
vis.plot_scores(img_target,score_rf45)
#vis.plot_scores(img_target,score_rf45.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf45.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf45.sum(1).unsqueeze(1), 45)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
layer = 6rf29_p = layer_vals[layer]rf29, mp_idx = F.max_pool2d(rf29_p, 2, 2, return_indices = True) # 1x64x5x5#del rf29_pscore_rf29, score_k45 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf45, rf29, rf45, me.rf45)#del score_rf45, valvis.stats(score_rf29)score_k45 = score_k45.sum(1).unsqueeze(1)print(score_k45.size())#vis.plot_scores(img_target, score_k45, figsize=(50,50))vis.plot_scores(0,score_k45.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k45.sum(1).unsqueeze(1), 45)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k45
vis.plot_scores(img_target,score_rf29)
#vis.plot_scores(img_target,score_rf29.sum(1).unsqueeze(1), figsize=(50,50), threshold=None)vis.plot_scores(0,score_rf29.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf29.sum(1).unsqueeze(1), 29)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
#del score_rf637, score_rf509, score_rf381, score_rf253, score_rf189#del score_rf125, score_rf93, score_rf61, score_rf45, score_mp#del rf637, rf509, rf381, rf253, rf189, rf125, rf93, rf61, rf45#del rf509_p, rf253_p, rf125_p, rf61_pscore_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf29, mp_idx)#del score_rf29, mp_idxlayer = 5rf21 = layer_vals[layer]score_rf21, score_k29 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf21, rf29_p, me.rf29)#del score_mp, rf45vis.stats(score_rf21)score_k29 = score_k29.sum(1).unsqueeze(1)print(score_k29.size())#vis.plot_scores(img_target, score_k29, figsize=(50,50))vis.plot_scores(0,score_k29.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k29, 29)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k29
vis.plot_scores(img_target, score_rf21)
#vis.plot_scores(img_target,score_rf21.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf21.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf21.sum(1).unsqueeze(1), 21)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
layer = 4rf13_p = layer_vals[layer]rf13, mp_idx = F.max_pool2d(rf13_p, 2, 2, return_indices = True) # 1x64x5x5score_rf13, score_k21 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf21, rf13, rf21, me.rf21)vis.stats(score_rf13)score_k21 = score_k21.sum(1).unsqueeze(1)print(score_k21.size())#vis.plot_scores(img_target, score_k21, figsize=(50,50))vis.plot_scores(0,score_k21.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k21, 21)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k21
vis.plot_scores(img_target, score_rf13)
#vis.plot_scores(img_target,score_rf13.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf13.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf13.sum(1).unsqueeze(1), 13)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
score_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf13, mp_idx)#del score_rf29, mp_idxlayer = 3rf9 = layer_vals[layer]score_rf9, score_k13 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf9, rf13_p, me.rf13)#del score_mp, rf45vis.stats(score_rf9)score_k13 = score_k13.sum(1).unsqueeze(1)print(score_k13.size())#vis.plot_scores(img_target, score_k13, figsize=(50,50))vis.plot_scores(0,score_rf13.sum(1).unsqueeze(0), figsize=(50,50))# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k13, 13)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k13
vis.plot_scores(img_target, score_rf9)
#vis.plot_scores(img_target,score_rf9.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf9.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf9.sum(1).unsqueeze(1), 9)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
layer = 2rf5_p = layer_vals[layer]rf5, mp_idx = F.max_pool2d(rf5_p, 2, 2, return_indices = True) # 1x64x5x5score_rf5, score_k9 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf9, rf5, rf9, me.rf9)vis.stats(score_rf5)score_k9 = score_k9.sum(1).unsqueeze(1)print(score_k9.size())#vis.plot_scores(img_target, score_k9, figsize=(50,50))vis.plot_scores(0,score_k9.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k9, 9)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k9
vis.plot_scores(img_target, score_rf5)
#vis.plot_scores(img_target,score_rf5.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf5.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf5.sum(1).unsqueeze(1), 5)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
#del score_rf21, score_rf13, score_rf9#del rf21, rf13, rf9#del rf29_p, rf13_pscore_mp = vis.propagate_score_through_maxpool_sz2x2_st2x2(score_rf5, mp_idx)#del score_rf29, mp_idxlayer = 1rf3 = layer_vals[layer]score_rf3, score_k5 = vis.propagate_score_through_conv2d_sz3_pad1(score_mp, rf3, rf5_p, me.rf5)#del score_mp, rf4vis.stats(score_rf3)score_k5 = score_k5.sum(1).unsqueeze(1)print(score_k5.size())#vis.plot_scores(img_target, score_k5, figsize=(50,50))vis.plot_scores(0,score_k5.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k5, 5)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k5
vis.plot_scores(img_target, score_rf3)
#vis.plot_scores(img_target,score_rf3.sum(1).unsqueeze(1), figsize=(50,50))vis.plot_scores(0,score_rf3.sum(1).unsqueeze(0), figsize=(50,50))
tmp = vis.map_scores_to_input_sz(score_rf3.sum(1).unsqueeze(1), 3)#vis.plot_scores(img_target,tmp, figsize=(50,50))vis.plot_scores(0,tmp.sum(1).unsqueeze(0), figsize=(50,50))del tmp
#del score_rf5, score_mp#del rf5_player = 0score_input, score_k3 = vis.propagate_score_through_conv2d_sz3_pad1(score_rf3, input, rf3, me.rf3)_, _, _, std = vis.stats(score_input)score_k3 = score_k3.sum(1).unsqueeze(1)print(score_k3.size())#vis.plot_scores(img_target, score_k3, figsize=(50,50))vis.plot_scores(0,score_k3.sum(1).unsqueeze(0), figsize=(50,50))
# map to input spacescore_kmapped = vis.map_scores_to_input_sz(score_k3, 3)#vis.plot_scores(img_target,score_kmapped, figsize=(50,50))vis.plot_scores(0,score_kmapped.sum(1).unsqueeze(0), figsize=(50,50))vis.stats(score_kmapped)score_accumulated_k.add_(score_kmapped)#del score_k3
## We can decide to sum score_accumulated_k or not!!#vis.plot_scores(img_target,score_input.sum(1).unsqueeze(1), figsize=(50,50), only_positive=False)vis.plot_scores(0,score_input.sum(1).unsqueeze(0), figsize=(50,50), only_positive=False)
print("Part of the score represented by last layer:")si = score_input.sum(3).sum(2).sum(1)print(si)sa = score_accumulated_k.sum(3).sum(2).sum(1).squeeze()print(sa)print(si+sa+me.lc[0].bias)vis.plot_scores(0,score_input.sum(1).unsqueeze(0), figsize=(50,50))print(score_input.sum(1).unsqueeze(0).sum(3).sum(2))score_total = score_input.sum(1).unsqueeze(0) + score_accumulated_k.squeeze().unsqueeze(0)vis.plot_scores(0, score_total, figsize=(50,50))print(score_total.sum(3).sum(2) + me.lc[0].bias)print(score_total.size())
# Test for checking that the sum of the scores is equal to the last layer score (for every target)nclasses = 5for t in range(nclasses): total_score = score_k637[t].sum() + score_k509[t].sum() + score_k381[t].sum() + score_k253[t].sum() + \ score_k189[t].sum() + score_k125[t].sum()+ score_k93[t].sum() + score_k61[t].sum() + \ score_k45[t].sum() + score_k29[t].sum() + score_k21[t].sum() + score_k13[t].sum() + \ score_k9[t].sum() + score_k5[t].sum() + score_k3[t].sum() + score_input[t].sum() + me.lc[0].bias[t] print("Class {} : {} == {} ".format(t, total_score.item(), output[0][t].item()))print("Score 637")print(score_k637.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 509")print(score_k509.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 381")print(score_k381.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 253")print(score_k253.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 189")print(score_k189.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 125")print(score_k125.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 93")print(score_k93.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 61")print(score_k61.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 45")print(score_k45.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 29")print(score_k29.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 21")print(score_k21.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 13")print(score_k13.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 9")print(score_k9.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 5")print(score_k5.sum(3).sum(2).sum(1).data.unsqueeze(1).t())print("Score 3")print(score_k3.sum(3).sum(2).sum(1).data.unsqueeze(1).t()) xxxxxxxxxxstd = torch.std(score_total.view(-1)).item()pc = pred_class.item()print("Score distribution for predicted class: {}".format(pc))vis.plot_scores(pc,score_total.squeeze().unsqueeze(1), figsize=(150,150), only_positive=True)#, threshold=2.0*std)
tmp = score_total.squeeze()[pc]vis.plot_final(tmp, path, only_positive=True, threshold=2*std, cmap=matplotlib.cm.PuOr, binary=True)
from IPython.display import Imagebase = os.path.splitext(os.path.basename(path))[0]command_img_animated = base + ".animated.gif"Image(url=command_img_animated)#tmp = score_total.squeeze()[pc]tmp = vis.map_scores_to_input_sz(score_rf3[pc].unsqueeze(0).sum(1).unsqueeze(1), 3).squeeze()vis.plot_final(tmp, path, only_positive=True)del tmp
#tmp = score_accumulated_k[pc].squeeze()#tmp = score_total.squeeze()[pc]tmp = score_input[pc].sum(0)vis.plot_final(tmp, path)#, only_positive=True, threshold=2*std, cmap=matplotlib.cm.PuOr, binary=True)
val = score_accumulated_k[pc].squeeze()maxval = val.data.max()minval = val.data.min()sumval = val.data.sum()stdval = val.data.std()meaval = val.data.mean()ne0 = torch.gt(torch.abs(val.data), 1e-5).sum()/(640.*640.)*100print("Score k for class {}:\nmax={}\nmin={}\nsum={}\nstd={}\nmean={}\nne0={}%" .format(pc, maxval, minval, sumval, stdval, meaval,ne0))val = score_input[pc].sum(0)maxval = val.data.max()minval = val.data.min()sumval = val.data.sum()stdval = val.data.std()meaval = val.data.mean()ne0 = torch.gt(torch.abs(val.data), 1e-5).sum()/(640.*640.)*100print("Score input for class {}:\nmax={}\nmin={}\nsum={}\nstd={}\nmean={}\nne0={}%" .format(pc, maxval, minval, sumval, stdval, meaval,ne0))val = score_total.squeeze()[pc]maxval = val.data.max()minval = val.data.min()sumval = val.data.sum()stdval = val.data.std()meaval = val.data.mean()ne0 = torch.gt(torch.abs(val.data), 1e-5).sum()/(640.*640.)*100print("Score total for class {}:\nmax={}\nmin={}\nsum={}\nstd={}\nmean={}\nne0={}%" .format(pc, maxval, minval, sumval, stdval, meaval,ne0))